{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Rental Listing Inquiries Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第三步：调整树的参数：max_depth & min_child_weight\n",
    "(将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "选择该数据集的特征工程已经处理好了，集中精力放在参数调优上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEKCAYAAAAxXHOuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGodJREFUeJzt3X+wHWWd5/H3hwCCAibClY1JMKyb\nUSNqhLshyuygiBCYXQn+mIJ1JGVRG8cNiqVlCZYalcEf5aA1mRFmYpElTDnGiD+IbtgYEXF15UfA\nCITI5E4AiUmROEEIOqKBz/7Rz4WTcHJPJ+lzT87N51XVdfp8++k+365bybe6n6eflm0iIiKaclCv\nE4iIiLElhSUiIhqVwhIREY1KYYmIiEalsERERKNSWCIiolEpLBER0agUloiIaFQKS0RENOrgXifQ\nC8ccc4ynTp3a6zQiIvrGHXfc8WvbA3XaHpCFZerUqaxevbrXaURE9A1JD9Ztm1thERHRqK4WFkmH\nSbpN0s8lrZX0yRK/RtL9ktaUZUaJS9JCSUOS7pJ0Ysux5kpaX5a5LfGTJN1d9lkoSd08p4iIGFm3\nb4U9AZxm+3FJhwA/lnRD2fYh29ft0v4sYFpZTgauAk6W9AJgATAIGLhD0nLbj5Q284BbgBXAbOAG\nIiKiJ7p6xeLK4+XrIWUZaZ7+c4Bry363AOMlTQTOBFbZ3laKySpgdtl2lO2fupr//1pgTtdOKCIi\nOup6H4ukcZLWAFuoisOtZdPl5XbXFyU9p8QmAQ+17L6xxEaKb2wTb5fHPEmrJa3eunXrPp9XRES0\n1/XCYvtJ2zOAycBMSScAlwIvA/4z8ALgw6V5u/4R70W8XR6LbA/aHhwYqDViLiIi9sKojQqz/Rvg\nh8Bs25vL7a4ngP8FzCzNNgJTWnabDGzqEJ/cJh4RET3S7VFhA5LGl/XDgdOBX5S+EcoIrjnAPWWX\n5cAFZXTYLOBR25uBlcAZkiZImgCcAaws27ZLmlWOdQFwfTfPKSIiRtbtUWETgSWSxlEVsWW2vyvp\nB5IGqG5lrQH+qrRfAZwNDAG/A94FYHubpMuA20u7T9neVtbfA1wDHE41GiwjwiIiekjVYKoDy+Dg\noPPkfcT+7ZS/O6XXKYx5P3nvT2q3lXSH7cE6bfPkfURENCqFJSIiGpXCEhERjUphiYiIRqWwRERE\no1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgiIqJRKSwR\nEdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY1KYYmIiEalsERERKO6WlgkHSbpNkk/l7RW0idL/HhJ\nt0paL+lrkg4t8eeU70Nl+9SWY11a4vdJOrMlPrvEhiRd0s3ziYiIzrp9xfIEcJrtVwMzgNmSZgGf\nA75oexrwCHBhaX8h8Ijt/wR8sbRD0nTgPOAVwGzgSknjJI0DvgScBUwHzi9tIyKiR7paWFx5vHw9\npCwGTgOuK/ElwJyyfk75Ttn+Rkkq8aW2n7B9PzAEzCzLkO0Ntv8ALC1tIyKiR7rex1KuLNYAW4BV\nwL8Cv7G9ozTZCEwq65OAhwDK9keBo1vju+yzu3hERPRI1wuL7SdtzwAmU11hvLxds/Kp3Wzb0/iz\nSJonabWk1Vu3bu2ceERE7JVRGxVm+zfAD4FZwHhJB5dNk4FNZX0jMAWgbH8+sK01vss+u4u3+/1F\ntgdtDw4MDDRxShER0Ua3R4UNSBpf1g8HTgfWATcBbyvN5gLXl/Xl5Ttl+w9su8TPK6PGjgemAbcB\ntwPTyiizQ6k6+Jd385wiImJkB3dusk8mAkvK6K2DgGW2vyvpXmCppL8GfgZcXdpfDfyTpCGqK5Xz\nAGyvlbQMuBfYAcy3/SSApIuAlcA4YLHttV0+p4iIGEFXC4vtu4DXtIlvoOpv2TX+e+DtuznW5cDl\nbeIrgBX7nGxERDQiT95HRESjUlgiIqJRKSwREdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY0a8TkW\nSR8YabvtLzSbTkRE9LtOD0geOSpZRETEmDFiYbH9ydFKJCIixoZafSyS/kTSjZLuKd9fJemj3U0t\nIiL6Ud3O+y8DlwJ/hKfnADuvW0lFRET/qltYnmv7tl1iO9q2jIiIA1rdwvJrSS+hvJ1R0tuAzV3L\nKiIi+lbdafPnA4uAl0n6FXA/8I6uZRUREX2rbmF50Pbpkp4HHGR7ezeTioiI/lX3Vtj9khZRva/+\n8S7mExERfa5uYXkp8H2qW2L3S/p7SX/avbQiIqJf1Sostv/d9jLbb6F61fBRwM1dzSwiIvpS7Uko\nJZ0q6UrgTuAw4C+6llVERPStWp33ku4H1gDLgA/Z/m1Xs4qIiL5V94rl1bbPtf3VPSkqkqZIuknS\nOklrJV1c4p+Q9CtJa8pydss+l0oaknSfpDNb4rNLbEjSJS3x4yXdKmm9pK9JOrRufhER0by6heU/\n7OVcYTuAD9p+OdWIsvmSppdtX7Q9oywrynGnU00V8wpgNnClpHGSxgFfAs4CpgPntxznc+VY04BH\ngAtrnlNERHRBV+cKs73Z9p1lfTuwDpg0wi7nAEttP2H7fmAImFmWIdsbbP8BWAqcI0nAacB1Zf8l\nwJya5xQREV0wanOFSZpKNaLs1hK6SNJdkhZLmlBik4CHWnbbWGK7ix8N/Mb2jl3iERHRI6MyV5ik\nI4BvAO+3/RhwFfASYEY5zhXDTdvs7r2It8thnqTVklZv3bq1buoREbGH9mWusL+ss6OkQ6iKylds\nfxPA9sMt278MfLd83QhMadl9MrCprLeL/xoYL+ngctXS2n4ntheVc2BwcLBt8YmIiH1X9wHJDbZP\nBwaAl9n+U9sPdNqv9IFcDayz/YWW+MSWZucC95T15cB5kp4j6XhgGnAbcDswrYwAO5Sqf2e5bQM3\nAW8r+88Frq9zThER0R0jXrFI+sBu4gC0FovdOAV4J3C3pDUl9hGqUV0zqG5bPQC8uxxvraRlwL1U\nfTjzbT9ZfvMiYCUwDlhse2053oeBpZL+GvgZVSGLiIge6XQr7Mh9ObjtH9O+H2TFCPtcDlzeJr6i\n3X62N1CNGouIiP3AiIXF9ifrHETSpbY/00xKERHRz2rPFdbB2xs6TkRE9LmmCku7210REXEAaqqw\nZPhuREQAuWKJiIiG1Soskk7pEPt6YxlFRERfq3vF8ncjxWx/upl0IiKi33V6QPK1wOuAgV0eljyK\n6kHFiIiInXR6QPJQ4IjSrvVhycd4ZhqViIiIp3V6QPJm4GZJ19h+EEDSQcARZZbiiIiIndTtY/mM\npKMkPY9qHq/7JH2oi3lFRESfqltYppcrlDlU83UdRzW5ZERExE7qFpZDyntV5gDX2/4jeSgyIiLa\nqFtY/pFqevvnAT+S9GKqDvyIiIid1HqDpO2FwMKW0IOS3tCdlCIiop/VffL+WElXS7qhfJ9O9bbG\niIiIndS9FXYN1dsbX1S+/wvw/m4kFBER/a1uYTnG9jLgKQDbO4Anu5ZVRET0rbqF5beSjqaMBJM0\nC3i0a1lFRETfqtV5D3wAWA68RNJPgAEypUtERLTRsbCUKVwOA04FXkr17pX7yrMsERERO+l4K8z2\nU8AVtnfYXmv7nrpFRdIUSTdJWidpraSLS/wFklZJWl8+J5S4JC2UNCTpLkknthxrbmm/XtLclvhJ\nku4u+yyUlJeORUT0UN0+lu9Jeute/Ke9A/ig7ZcDs4D5ZajyJcCNtqcBN5bvAGcB08oyD7gKqkIE\nLABOBmYCC4aLUWkzr2W/2XuYY0RENKhuYfkA1Vsi/yDpMUnbJXV88t72Ztt3lvXtwDpgEnAOsKQ0\nW0I1VQwlfq0rtwDjJU0EzgRW2d5m+xFgFTC7bDvK9k9tG7i25VgREdEDdZ+8P7Jzq5FJmgq8BrgV\nONb25nLszZJeWJpNAh5q2W1jiY0U39gmHhERPVL3yXtJ+ktJHyvfp0iaWfdHJB0BfAN4f4f3uLS7\n1ea9iLfLYZ6k1ZJWb926tVPKERGxl+reCrsSeC3w38v3x4Ev1dmxzIr8DeArtr9Zwg+X21iUzy0l\nvhGY0rL7ZGBTh/jkNvFnsb3I9qDtwYGBgTqpR0TEXqhbWE62PR/4PUDp5zi0006ls/9qYJ3tL7Rs\nWs4zc43NBa5viV9QrpBmAY+WW2YrgTMkTSid9mcAK8u27ZJmld+6oOVYERHRA3UfkPyjpHE88+T9\nAGV6lw5OoXoh2N2S1pTYR4DPAsskXQj8Enh72bYCOBsYAn4HvAvA9jZJlwG3l3afsr2trL+Hai6z\nw4EbyhIRET1St7AsBL4FvFDS5VRP3X+s0062f0z7fhCAN7Zpb2D+bo61GFjcJr4aOKFTLhERMTrq\njgr7iqQ7qIqBgDm213U1s4iI6Eu1Coukf7L9TuAXbWIRERFPq9t5/4rWL6W/5aTm04mIiH43YmGR\ndKmk7cCrWp643041PDijryIi4llGLCy2P1Oeuv+87aNsH1mWo21fOko5RkREH6nbeX+ppEnAi1v3\nsf2jbiUWERH9qW7n/WeB84B7eeaVxAZSWCIiYid1n2M5F3ip7Se6mUxERPS/uqPCNgCHdDORiIgY\nG+pesfwOWCPpRuDpqxbb7+tKVhER0bfqFpblZYmIiBhR3VFhSzq3ioiI6FBYJC2z/ReS7qbNC7Rs\nv6prmUVERF/qdMVycfn8r91OJCIixoYRC0vLe+kfHJ10IiKi39UdbhwREVFLCktERDSq0+zGN5bP\nz41OOhER0e86dd5PlHQq8GZJS9nlNcO27+xaZhER0Zc6FZaPA5cAk4Ev7LLNwGndSCoiIvpXp1Fh\n1wHXSfqY7ctGKaeIiOhjdZ+8v0zSm4E/K6Ef2v5u99KKiIh+VWtUmKTPUD0seW9ZLi6xTvstlrRF\n0j0tsU9I+pWkNWU5u2XbpZKGJN0n6cyW+OwSG5J0SUv8eEm3Slov6WuSDq132hER0S11hxv/OfAm\n24ttLwZml1gn15S2u/qi7RllWQEgaTrVy8ReUfa5UtI4SeOALwFnAdOB80tbgM+VY00DHgEurHk+\nERHRJXvyHMv4lvXn19mhvLp4W83jnwMstf2E7fuBIWBmWYZsb7D9B2ApcI4kUQ0euK7svwSYU/O3\nIiKiS+oWls8AP5N0jaQlwB3Ap/fhdy+SdFe5VTahxCYBD7W02Vhiu4sfDfzG9o5d4m1JmidptaTV\nW7du3YfUIyJiJLUKi+2vArOAb5bltbaX7uVvXgW8BJgBbAauKHG1aeu9iLdle5HtQduDAwMDe5Zx\nRETUVvdFX8MTUu7zy75sPzy8LunLwPDoso3AlJamk4FNZb1d/NfAeEkHl6uW1vYREdEjoz5XmKSJ\nLV/PBYZHjC0HzpP0HEnHA9OA24DbgWllBNihVB38y20buAl4W9l/LnD9aJxDRETsXu0rlr0h6avA\n64FjJG0EFgCvlzSD6rbVA8C7AWyvlbSMajjzDmC+7SfLcS4CVgLjgMW215af+DCwVNJfAz8Dru7m\n+URERGcdC4ukg4C7bJ+wpwe3fX6b8G7/87d9OXB5m/gKYEWb+AaqUWMREbGf6HgrzPZTwM8lHTcK\n+URERJ+reytsIrBW0m3Ab4eDtt/clawiIqJv1S0sn+xqFhERMWbUnYTyZkkvBqbZ/r6k51J1pEdE\nROyk7iSU/4Nq6pR/LKFJwLe7lVRERPSvus+xzAdOAR4DsL0eeGG3koqIiP5Vt7A8USaABEDSwYww\nfUpERBy46haWmyV9BDhc0puArwPf6V5aERHRr+oWlkuArcDdVE/KrwA+2q2kIiKif9UdFfZUmS7/\nVqpbYPeVuboiIiJ2UquwSPpz4B+Af6Warv54Se+2fUM3k4uIiP5T9wHJK4A32B4CkPQS4H8DKSwR\nEbGTun0sW4aLSrEB2NKFfCIios+NeMUi6S1lda2kFcAyqj6Wt1O9JyUiImInnW6F/beW9YeBU8v6\nVmDCs5tHRMSBbsTCYvtdo5VIRESMDXVHhR0PvBeY2rpPps2PiIhd1R0V9m2qNz9+B3iqe+lENOeX\nn3plr1M4IBz38bt7nULsZ+oWlt/bXtjVTCIiYkyoW1j+VtIC4HvAE8NB23d2JauIiOhbdQvLK4F3\nAqfxzK0wl+8RERFPq/uA5LnAf7R9qu03lKVjUZG0WNIWSfe0xF4gaZWk9eVzQolL0kJJQ5LuknRi\nyz5zS/v1kua2xE+SdHfZZ6Ek1T/1iIjohrqF5efA+L04/jXA7F1ilwA32p4G3Fi+A5wFTCvLPOAq\nqAoRsAA4GZgJLBguRqXNvJb9dv2tiIgYZXULy7HALyStlLR8eOm0k+0fAdt2CZ8DLCnrS4A5LfFr\nXbkFGC9pInAmsMr2NtuPAKuA2WXbUbZ/WmZavrblWBER0SN1+1gWNPibx9reDGB7s6ThVxxPAh5q\nabexxEaKb2wTb0vSPKqrG4477rh9PIWIiNiduu9jubnbiVBNx/+sn96LeFu2FwGLAAYHB/MumYiI\nLql1K0zSdkmPleX3kp6U9Nhe/ubD5TYW5XN4luSNwJSWdpOBTR3ik9vEIyKih2oVFttH2j6qLIcB\nbwX+fi9/czkwPLJrLnB9S/yCMjpsFvBouWW2EjhD0oTSaX8GsLJs2y5pVhkNdkHLsSIiokfq9rHs\nxPa3JV3SqZ2krwKvB46RtJGqr+azwDJJFwK/pJqCH2AFcDYwBPwOeFf5rW2SLuOZafo/ZXt4QMB7\nqEaeHU710rG8eCwiosfqTkL5lpavBwGDjNCfMcz2+bvZ9MY2bQ3M381xFgOL28RXAyd0yiMiIkZP\n3SuW1vey7AAeoBoeHBERsZO6o8LyXpaIiKil06uJPz7CZtu+rOF8IiKiz3W6Yvltm9jzgAuBo4EU\nloiI2EmnVxNfMbwu6UjgYqrRWkuBK3a3X0REHLg69rGUSSA/ALyDam6vE8ucXREREc/SqY/l88Bb\nqKZCeaXtx0clq4iI6Fudnrz/IPAi4KPAppZpXbbvw5QuERExhnXqY6k7rX5ERARQ/30sERERtaSw\nREREo1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgiIqJR\nKSwREdGonhUWSQ9IulvSGkmrS+wFklZJWl8+J5S4JC2UNCTpLkknthxnbmm/XtLcXp1PRERUen3F\n8gbbM2wPlu+XADfangbcWL4DnAVMK8s84Cp4+iVkC4CTgZnAguFiFBERvdHrwrKrc6jeUkn5nNMS\nv9aVW4DxkiYCZwKrbG8rb7VcBcwe7aQjIuIZvSwsBr4n6Q5J80rsWNubAcrnC0t8EvBQy74bS2x3\n8YiI6JGO77zvolNsb5L0QmCVpF+M0FZtYh4h/uwDVMVrHsBxxx23p7lGRERNPbtisb2pfG4BvkXV\nR/JwucVF+dxSmm8EprTsPhnYNEK83e8tsj1oe3BgYKDJU4mIiBY9KSySnifpyOF14AzgHmA5MDyy\nay5wfVlfDlxQRofNAh4tt8pWAmdImlA67c8osYiI6JFe3Qo7FviWpOEc/tn2/5F0O7BM0oXAL4G3\nl/YrgLOBIeB3wLsAbG+TdBlwe2n3KdvbRu80IiJiVz0pLLY3AK9uE/834I1t4gbm7+ZYi4HFTecY\nERF7Z38bbhwREX0uhSUiIhrVy+HGfeGkD13b6xTGvDs+f0GvU4iIBuWKJSIiGpXCEhERjUphiYiI\nRqWwREREo1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgi\nIqJRKSwREdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY0aE4VF0mxJ90kaknRJr/OJiDiQ9X1hkTQO\n+BJwFjAdOF/S9N5mFRFx4Or7wgLMBIZsb7D9B2ApcE6Pc4qIOGCNhcIyCXio5fvGEouIiB44uNcJ\nNEBtYn5WI2keMK98fVzSfV3NqneOAX7d6yT2hP5mbq9T2J/03d+PBe3+CR6w+urvp/ft0d/uxXUb\njoXCshGY0vJ9MrBp10a2FwGLRiupXpG02vZgr/OIvZO/X3/L368yFm6F3Q5Mk3S8pEOB84DlPc4p\nIuKA1fdXLLZ3SLoIWAmMAxbbXtvjtCIiDlh9X1gAbK8AVvQ6j/3EmL/dN8bl79ff8vcDZD+rnzsi\nImKvjYU+loiI2I+ksIwhmdqmf0laLGmLpHt6nUvsGUlTJN0kaZ2ktZIu7nVOvZZbYWNEmdrmX4A3\nUQ3Bvh043/a9PU0sapH0Z8DjwLW2T+h1PlGfpInARNt3SjoSuAOYcyD/28sVy9iRqW36mO0fAdt6\nnUfsOdubbd9Z1rcD6zjAZ/9IYRk7MrVNRI9Jmgq8Bri1t5n0VgrL2FFrapuI6A5JRwDfAN5v+7Fe\n59NLKSxjR62pbSKieZIOoSoqX7H9zV7n02spLGNHpraJ6AFJAq4G1tn+Qq/z2R+ksIwRtncAw1Pb\nrAOWZWqb/iHpq8BPgZdK2ijpwl7nFLWdArwTOE3SmrKc3eukeinDjSMiolG5YomIiEalsERERKNS\nWCIiolEpLBER0agUloiIaFQKS0TDJI2X9D9H4XdeL+l13f6diD2VwhLRvPFA7cKiyt78W3w9kMIS\n+508xxLRMEnDM0vfB9wEvAqYABwCfNT29WWywhvK9tcCc4DTgQ9TTcWzHnjC9kWSBoB/AI4rP/F+\n4FfALcCTwFbgvbb/72icX0QnKSwRDStF47u2T5B0MPBc249JOoaqGEwDXgxsAF5n+xZJLwL+H3Ai\nsB34AfDzUlj+GbjS9o8lHQestP1ySZ8AHrf9N6N9jhEjObjXCUSMcQI+XV7k9RTVqwyOLdsetH1L\nWZ8J3Gx7G4CkrwN/UradDkyvpqQC4KjyQqmI/VIKS0R3vQMYAE6y/UdJDwCHlW2/bWnX7rUHww4C\nXmv731uDLYUmYr+SzvuI5m0Hhq8ong9sKUXlDVS3wNq5DThV0oRy++ytLdu+RzXBKACSZrT5nYj9\nRgpLRMNs/xvwE0n3ADOAQUmrqa5efrGbfX4FfJrqzYPfB+4FHi2b31eOcZeke4G/KvHvAOeW2XT/\nS9dOKGIPpfM+Yj8h6Qjbj5crlm8Bi21/q9d5ReypXLFE7D8+IWkNcA9wP/DtHucTsVdyxRIREY3K\nFUtERDQqhSUiIhqVwhIREY1KYYmIiEalsERERKNSWCIiolH/H0mmMnB4UroEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x83dfef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of interest_level');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀，所以交叉验证时也考虑各类样本按比例抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二轮参数调整得到的n_estimators最优值（219），max_depth为7，min_child_weight为5，其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [6, 7, 8], 'min_child_weight': [4, 5, 6]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [6,7,8]\n",
    "min_child_weight = [4,5,6]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58840, std: 0.00287, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.58863, std: 0.00306, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.58863, std: 0.00389, params: {'max_depth': 6, 'min_child_weight': 6},\n",
       "  mean: -0.58924, std: 0.00449, params: {'max_depth': 7, 'min_child_weight': 4},\n",
       "  mean: -0.58880, std: 0.00361, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.58992, std: 0.00375, params: {'max_depth': 7, 'min_child_weight': 6},\n",
       "  mean: -0.59380, std: 0.00471, params: {'max_depth': 8, 'min_child_weight': 4},\n",
       "  mean: -0.59243, std: 0.00381, params: {'max_depth': 8, 'min_child_weight': 5},\n",
       "  mean: -0.59200, std: 0.00446, params: {'max_depth': 8, 'min_child_weight': 6}],\n",
       " {'max_depth': 6, 'min_child_weight': 4},\n",
       " -0.58839841822716221)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 454.19079995,  456.84860001,  452.40079999,  503.95220003,\n",
       "         503.6178    ,  504.08999996,  559.89919996,  557.44739995,\n",
       "         521.08299994]),\n",
       " 'mean_score_time': array([ 1.27820005,  1.35639997,  1.2598    ,  1.66559997,  1.56860003,\n",
       "         1.54099998,  2.225     ,  2.0092    ,  1.68440003]),\n",
       " 'mean_test_score': array([-0.58839842, -0.58862871, -0.58863239, -0.58923501, -0.58879754,\n",
       "        -0.58991791, -0.59380009, -0.59243408, -0.59200358]),\n",
       " 'mean_train_score': array([-0.48268752, -0.48579975, -0.48892464, -0.43937799, -0.44481989,\n",
       "        -0.44937285, -0.39604813, -0.40428537, -0.41191113]),\n",
       " 'param_max_depth': masked_array(data = [6 6 6 7 7 7 8 8 8],\n",
       "              mask = [False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_min_child_weight': masked_array(data = [4 5 6 4 5 6 4 5 6],\n",
       "              mask = [False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'max_depth': 6, 'min_child_weight': 4},\n",
       "  {'max_depth': 6, 'min_child_weight': 5},\n",
       "  {'max_depth': 6, 'min_child_weight': 6},\n",
       "  {'max_depth': 7, 'min_child_weight': 4},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 6},\n",
       "  {'max_depth': 8, 'min_child_weight': 4},\n",
       "  {'max_depth': 8, 'min_child_weight': 5},\n",
       "  {'max_depth': 8, 'min_child_weight': 6}],\n",
       " 'rank_test_score': array([1, 2, 3, 5, 4, 6, 9, 8, 7]),\n",
       " 'split0_test_score': array([-0.58380151, -0.58346045, -0.5817826 , -0.58227063, -0.58301462,\n",
       "        -0.58309901, -0.5861502 , -0.58547741, -0.58389605]),\n",
       " 'split0_train_score': array([-0.48390029, -0.48698385, -0.4900867 , -0.44065444, -0.4476738 ,\n",
       "        -0.45217294, -0.39662615, -0.40402716, -0.41244834]),\n",
       " 'split1_test_score': array([-0.58726781, -0.5884751 , -0.58797838, -0.58907479, -0.58712017,\n",
       "        -0.58964942, -0.59062045, -0.59117161, -0.5907653 ]),\n",
       " 'split1_train_score': array([-0.48165305, -0.48559293, -0.48917151, -0.43811254, -0.44358349,\n",
       "        -0.44926503, -0.39867053, -0.40515068, -0.41243351]),\n",
       " 'split2_test_score': array([-0.58905388, -0.58937444, -0.59015118, -0.58962561, -0.58981765,\n",
       "        -0.59167966, -0.59590752, -0.59554168, -0.59413821]),\n",
       " 'split2_train_score': array([-0.4817981 , -0.48501912, -0.49017183, -0.43764882, -0.44368944,\n",
       "        -0.44828809, -0.39678072, -0.40583275, -0.41098881]),\n",
       " 'split3_test_score': array([-0.58929665, -0.5887956 , -0.58961478, -0.58878071, -0.59015468,\n",
       "        -0.59080963, -0.59814019, -0.59507629, -0.59467867]),\n",
       " 'split3_train_score': array([-0.48338147, -0.48510524, -0.48721044, -0.43882942, -0.44348998,\n",
       "        -0.44811014, -0.39367747, -0.40431089, -0.41290559]),\n",
       " 'split4_test_score': array([-0.59257351, -0.59303929, -0.59363655, -0.59642549, -0.59388213,\n",
       "        -0.59435318, -0.59818342, -0.59490414, -0.59654104]),\n",
       " 'split4_train_score': array([-0.48270468, -0.48629761, -0.48798272, -0.44164475, -0.44566274,\n",
       "        -0.44902805, -0.39448578, -0.40210536, -0.41077939]),\n",
       " 'std_fit_time': array([ 12.18449055,   8.94154221,   4.6304671 ,  11.20704104,\n",
       "         11.399301  ,  12.09496771,  13.45838123,   7.65489113,  64.64655929]),\n",
       " 'std_score_time': array([ 0.03024173,  0.11165768,  0.06276108,  0.21352157,  0.02336331,\n",
       "         0.06489372,  0.23485566,  0.03793888,  0.33204673]),\n",
       " 'std_test_score': array([ 0.00286618,  0.00305838,  0.00388993,  0.0044851 ,  0.0036052 ,\n",
       "         0.00374535,  0.0047143 ,  0.00381395,  0.00446243]),\n",
       " 'std_train_score': array([ 0.00087337,  0.00074625,  0.00116555,  0.00152708,  0.00163893,\n",
       "         0.00146575,  0.00177813,  0.00126198,  0.00085813])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588398 using {'max_depth': 6, 'min_child_weight': 4}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAELCAYAAADgPECFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNXd+PHPNzshLIEkQIAkgAsg\nO1FEyiKQqNWiPKJ1rRuPVlu3X6XgY5+qXZ5GSt36tC6P1rWtWu1CXdqETRRFDAiKiAuYRAhLCGsS\nErJ8f3/cO8mQzCSTZTJJ+L5fr3nNzLnn3vnOeOWbc+6554iqYowxxgRLWKgDMMYY07VZojHGGBNU\nlmiMMcYElSUaY4wxQWWJxhhjTFBZojHGGBNUlmiMMcYElSUaY4wxQWWJxhhjTFBFhDqAjiAhIUHT\n0tJCHYYxxnQq69ev36eqiU3Vs0QDpKWlkZubG+owjDGmUxGR/EDqWdeZMcaYoLJEY4wxJqgs0Rhj\njAkqSzTGGGOCyhKNMcaYoLJEY4wxJqgs0RhjjAkqu4+mFfKLS1n95T56REcQFx1BXIzz3DMmsvZ1\nVITlcmPMic0STSts2nGI//775kbrREWEOYkoJoIebvKJi46se+2W96hNVJHERbtlXnWiI8Lb6VsZ\nY0zbskTTCuee1p8P75lNSUUVJeVVHKmo5Ei587qkwnkcKa/iSHmlV50qCg8e5UhFpfO+vIqqGm3y\ns6LCw+olq7rXPbxaUMdvPz6hxUVHEB0Rhoi0w69jjDEOSzStEBURRmKPaBJ7RLf4GKpKRVVNXSJy\nE1bDZFVFiVf5kfIqCg+We9WppLK66YQVGS5OYvLq7uvhSVAx9VpbXuU9oo9PZpawjDGBskQTYiJC\nTGQ4MZHhJMS1PGEBVFRV1yYrTzJyElGlV7KqOi5ZHSmvZPfhcr4qqmtxHauqafKzIsPFq6UUWS9Z\n1SUw7xZXD6+Wlae1ZQnLmK7PEk0XEh0RTnRcOH3bKGEdl6y8WlpHGiQrJ5ntOVLOtiKn/HB5YAkr\nIkzqklO9gRTHtbaiI4iLiazXZVjX2oqJtIRlTEdlicY00JYJq7Si2uf1q7pkVVfuKSs6UsH2opLa\nRFYRQMIKD5PjuvvqJ6ee9a5V9fDuJvRqbXWLDLeEZUwbs0RjgiY6IpzoiHD6dI9q1XGOVdVQWuFp\nKTV+/cq7tbWv5Bh5xWW128srm05YYYLfgRTHlR3X4mo4GCM2yhKWMR6WaEyHFxURRlREFPGtTFiV\n1TUNuwSbuH5VUlHF/tJjFBSX1Saxo5XVTX6Wd8Kq35LyHubecDDG8a0tS1imK7BEY04YkeFhxHdv\nm4RVelyycpKS72TltrYqqjhYdoxvDpTVDtgIJGGJQJiPRFO/xFcukvq1fNZp/nF816l/nKZjDiwe\nH8dp+ms12M93naYDCuyz6tdpm5h9aXCcNvjvdfkZKdw0fViTn90almiMaabI8DB6x0bRO7Z1Cauq\nuobSimpnkIUncZUff/2qpLyK+rdZKccXqI9R7fWLfNdpcOAAjtOwUv0iX4PsG9Zp+ji+1P/8tvqs\nQI7jq1aD4wTwO4f8v1e99wN6d2t4oDZmicaYEIkID6NXbBi9YiNDHYoxQWUTcRljjAkqSzTGGGOC\nyhKNMcaYoLJrNMaEkiocPQBHdsHhXXCk0H12H2XF0G8UDDsbhkyDbvGhjtiYZrNEY0ywVB2Dkt31\nEohXIjlcCEd2Q9XRhvvG9oUeydCtN3zyKqx/BiQMksfD0LNh6AwYPAkiWjfyzZj2YInGmOby2wqp\n91y2r+G+4dHQc4CTRAZOgB4DnIenrKf7PsJr+p/qSti5HrathO0r4d2H4J0lEBkLqVOc1s7QsyFp\nhO+bJowJMfE1zvpEk56errm5uaEOw3QEVcfquq0OF3q9rldWVd5wX08rxJMseiY3fO4W3/pkUH4I\n8t6tSzzFXznlcf2dls4wt8XTo3/rPseYJojIelVNb6qetWjMiaGtWyH1k0iP/se3QoIpphcMP995\nABz8xkk421bCl9nw8UtOedJIp6Uz7GxIPQuiurdPfMbUE5IWjYj0AV4G0oA84FJVPeCjXjXwifu2\nQFXnuOWzgF/jjJorAa5V1a9EJBX4A5AI7AeuUtUdTcVjLZpOzlcrpPbZ68K6z1ZIQsNuq2C0QtpL\nTQ3s/rgu8RSsheoKCI9yrukMneEkngHjIMyWBzetE2iLJlSJZjGwX1WzRGQREK+qC33UK1HVOB/l\nXwAXqupnInILcIaqXisifwFeV9XnRGQmcJ2qXt1UPJZoOihPK6RBF1YzWyH+urLasxUSKsfKoOB9\nN/Gsgj3u323d4p1RbJ4WT3xaKKM0nVRH7zq7EJjhvn4OWAU0SDSNUKCn+7oXUOi+Hgnc6b5eCfy9\nNUGaIGqyFeIZkdVEK2TgRK9k4pVUOlMrJJiiYuGkWc4DoGQvbH+7rsWz5R9OefyQukEFQ6Y5o92M\naSOhatEcVNXeXu8PqGqDGwREpArYCFQBWar6d7d8Kk4SOQocBs5U1cMi8ifgA1V9RET+A3gNSFDV\nYh/HvhG4ESAlJWVifn5+m3/PE1L9Vshxz7sbb4VExHi1OPqfuK2Q9qIK+76oG1SQ9y4cK3GHUU+o\nSzyDTrdh1MankHedicgywNewl3uA5wJMNMmqWigiQ4EVwCxV3SYifwUeUNUPRGQBcKqqzheRZOB/\ngSHAauBi4DRVPdRYrNZ1FqCqCjdZtLIV4vPZWiEhV10JOz6sSzw714PWQGR3SPtWXeJJPNX+Oxmg\nAySaRj9U5HNghqruEpEBwCpVPbWJfZ4FXgfeBtaq6jC3PAX4l6qOrFc/DtiqqoOaiueETzSNtkK8\nkkhZg4ZhvVZI/ftBPC0Ta4V0SkcPQt47dYln/3anvMeAums7Q2dAXFIoozQh1NGv0SwFrgGy3Od/\n1K8gIvFAmapWiEgCMAVYDBwAeonIKar6BZABfObuk4AzyKAGuBtnBNqJrarC//QmTbVCuic6SaJn\nMgyyayEnnG69YcR3nAfAgXwn4WxfBV+8BZv+5JQnnVbX2kk9y7kuZIyXULVo+gKvAClAAXCJqu4X\nkXTg+2432FnAE0ANzjDmh1X1aXf/ucDP3G0HgOtVdbuIzAN+hTNYYDXwA1WtaCqeTtmiCWYrpOcA\n5+Y/65c3/tTUwO5Nda2dgrVQfaxuGLUn8QwYB2E2d29X1aG7zjqaDpdo/LVC6l9Yr/aRQ7sn+rmI\nPsBaISZ4jpVBwXtu4lkFezY75d3iYcj0usQTnxrSME3b6uhdZycmVSjbX9dd1dJWyKDTrRViOpao\nWDhptvMAdxj1qroWzxb3ToM+Q+uu76RNtWHUJwhr0dBGLZpgtEK8k0hMb2uFmM5JFYo+r7t3J+9d\nqCx1hlEPnFiXeAadDuG2rHVnYl1nzdDiRPPVcsi5t5FWSDc/05tYK8ScwKqOOcOoPYmncIMzjDoq\nzhlG7Uk8CafYH1cdnHWdtYeoOOg1EAalWyvEmEBFREHaFOcx8yfOoJav36lLPF/8y6nXI7nu2s7Q\nGRCXGMqoTStYi4YOOBjAmBPZgby6azvb34byg055v9EwbEbdMOrIbqGM0mBdZ81iicaYDqqmGnZt\nrBvNVrAWaiqdSVNTzqxr8fQfY8OoQ8ASTTNYojGmkzhWCvnv13Wz7f3UKY/te/ww6t6DQxvnCcKu\n0Rhjup6o7nDybOcBcGSP09LxJJ5P/+qU9z2p7trOkKnOYnEmZKxFg7VojOkSVKFoq9ds1GvcYdTh\nzjDq2tmo020YdRuxrrNmsERjTBdUdQx2rKtLPIUfucOoexw/G3XCyTY6tIUs0TSDJRpjTgBHD8DX\nq+sSz4E8p7znoLolrofOgO4JIQuxs7FrNMYY461bPIy80HkA7P+67trO1n/Cxhed8v6j624aTZls\nw6jbgLVosBaNMSe8mmoo3AjbV8C2VfDNB84w6ogYZxi1J/H0G23DqL1Y11kzWKIxxhynogTy36tr\n8RR95pTHJsDQ6XWJp1eT6yp2adZ1ZowxLRUdB6dkOg9wJsndvqpuKPXm15zyvifXDSpI+xbE9AxV\nxB2atWiwFo0xphlUYe9nda2d/DVQWeYMox50el3iGTgRwrv23/LWddYMlmiMMS1WVQHfrPOajfoj\nQCG6p7Pmjifx9B3W5YZRW6JpBks0xpg2U7bfGUbtSTwH853yXoPrhlEPmQHd+4YwyLZh12iMMSYU\nYvvAaRc5D4D92+vu3flsKXz0glPef0xdaydlMkTGhC7mILMWDdaiMca0k5pqp2vNk3i+Wec1jHpy\nXeLpN6pTDKO2rrNmsERjjAmJihJnMIEn8RRtdcpjE7xmKzjbWWCxA7KuM2OM6eii4+CUc5wHwOFC\nZwi1Z/2dza865Qmn1N27k/YtiO4RqohbxFo0WIvGGNMBqcKeT72GUb8HVUchLMIZRu1JPMkTQjaM\n2rrOmsESjTGmw6ssd6bG8SSeXZtwhlH3ctbcGToDhs2EPkPbbRi1JZpmsERjjOl0yvYfP1vBwQKn\nvFcKDJtRt/BbbJ+ghWCJphks0RhjOjVVZxi1p7Xz9TtQcQgQGDDWaxj1mRAR3WYfa4mmGSzRGGO6\nlOoqZxi1J/HsWAc1VRDRDVIne81GPapV3WwdOtGISB/gZSANyAMuVdUDPupVA5+4bwtUdY5bPhNY\nAkQB64EbVLVKRAR4BPg2UAZcq6obmorHEo0xpkurOOIsbe1JPPs+d8q7J8LUH8GZN7fosB19ePMi\nYLmqZonIIvf9Qh/1jqrqOO8CEQkDngNmqeoXIvIz4BrgaeA84GT3MQl4zH02xpgTV3QPOPVc5wFw\naGfdtZ3Y4E+FE6pbTy/ESRa4zxc1Y9++QIWqfuG+zwEu9jru8+pYC/QWkQFtEbAxxnQZvQbC+Cvh\n4qdgzKVB/7hQJZp+qroLwH1O8lMvRkRyRWStiHiS0T4gUkQ8zbV5wGD39UDgG6/9d7hlxhhjQiRo\nXWcisgzo72PTPc04TIqqForIUGCFiHyiqttE5DLgIRGJBrKBKs/H+jiGz4tQInIjcCNASkpKM0Iy\nxhjTHEFLNKo62982EdkjIgNUdZfbtbXXzzEK3eftIrIKGA9sU9X3ganusTKBU9xddlDXugEYBBT6\nOfaTwJPgDAZoxlczxhjTDKHqOluKcwEf9/kf9SuISLzbYkFEEoApwBb3fZL7HI0ziOBxr+N+Txxn\nAoc8XXTGGGNCI1SJJgvIEJEvgQz3PSKSLiJPuXVGALkisglYCWSp6hZ32wIR+Qz4GPinqq5wy98E\ntgNfAf8H3NIu38YYY4xfdsMmdh+NMca0RKD30XT8lXWMMcZ0apZojDHGBJUlGmOMMUFlicYYY0xQ\nWaIxxhgTVJZojDHGBJUlGmOMMUFlicYYY0xQWaIxxhgTVJZojDHGBJUlGmOMMUHVrETjzqg8JljB\nGGOM6XqaTDQiskpEeopIH2AT8IyIPBj80IwxxnQFgbRoeqnqYeA/gGdUdSLgd1EzY4wxxlsgiSbC\nXQXzUuD1IMdjjDGmiwkk0fwM+Dfwlap+KCJDgS+DG5YxxpiuIqKpCqr6F+AvXu+3AxcHMyhjjDFd\nRyCDARa7gwEiRWS5iOwTkavaIzhjjDGdXyBdZ5nuYIALgB3AKcCCoEZljDGmywgk0US6z98G/qyq\n+4MYjzHGmC6myWs0wD9FZCtwFLhFRBKB8uCGZYwxpqtoskWjqouAyUC6qlYCpcCFwQ7MGGNM19Bk\ni0ZEIoGrgWkiAvA28HiQ4zLGGNNFBNJ19hjOdZrfu++vdsvmBysoY4wxXUcgieZ0VR3r9X6FiGwK\nVkDGGGO6lkBGnVWLyDDPG3dmgOrghWSMMaYrCaRFswBYKSLbAQFSgeuCGpUxxpguI5ApaJaLyMnA\nqTiJZiswLtiBGWOM6RoCWvhMVStU9WNV3aSqFXjNfdYSItJHRHJE5Ev3Od5PvWoR2eg+lnqVzxSR\nDSKyWUSeE5EIt3y4iLwvIhUicldrYjTGGNM2WrqUs7TycxcBy1X1ZGC5+96Xo6o6zn3MARCRMOA5\n4DJVHQXkA9e49fcDtwFLWhmfMcaYNtLSRKOt/NwLcZIF7vNFzdi3L1Chql+473NwZ5NW1b2q+iFQ\n2cr4jDHGtBG/12hE5J/4TiiC8499a/RT1V0AqrpLRJL81IsRkVygCshS1b8D+4BIEUlX1VxgHjC4\nlfEYY4wJksYGAzTW/dRk15SILAP6+9h0T1P7eklR1UJ3SPUKEflEVbeJyGXAQyISDWTjJKJmEZEb\ngRsBUlJSmru7McaYAPlNNKr6dmsOrKqz/W0TkT0iMsBtzQwA9vo5RqH7vF1EVgHjgW2q+j4w1T1W\nJs7SBc2N70ngSYD09PTWdgUaY4zxo6XXaFprKXUX8K8B/lG/gojEuy0WRCQBmAJscd8nuc/RwEJs\n7jVjjOmwArlhMxiygFdE5AagALgEQETSge+r6nxgBPCEiNTgJMQsVd3i7r9ARC5wyx9T1RXu/v2B\nXKAnUCMidwAj3YXbjDHGhICoWq9Renq65ubmhjoMY4zpVERkvaqmN1UvkGUCfI0+O4TTcnhCVW0R\nNGOMMX4Fco1mO1AC/J/7OAzswbkA/3/BC80YY0xXEMg1mvGqOs3r/T9FZLWqThORT4MVmDHGmK4h\nkBZNoojU3mjivk5w3x4LSlTGGGO6jEBaND8C3hWRbTizAgwBbhGR7tRNI2OMMcb4FMgyAW+6ywQM\nx10mwGsAwMPBDM4YY0znF8ios0jgJsBznWaViDyhqjZxpTHGmCYF0nX2GBAJ/N59f7VbNj9YQRlj\njOk6Akk0p6vqWK/3K0RkU7ACMsYY07UEMuqsWkSGed64MylXBy8kY4wxXUkgLZoFwEoR2Y4zGCAV\nuC6oURljjOkyAhl1ttwddXYq7qgzYFywAzPGGNM1BDR7s6pWAB973ovIXwBbLcwYY0yTWroejbRp\nFMYYY7qsliYaW1vAGGNMQPx2nflZHgCc1kzfoEVkjDGmS2nsGs2SFm4zxhhjavlNNKr6dnsGYowx\npmtq6TUaY4wxJiCWaIwxxgRVsxKNiPQPViDGGGO6pua2aN4MShTGGGO6rOYmGrtR0xhjTLM0N9H8\nX1CiMMYY02U1K9Go6u+brmWMMcbUsVFnxhhjgsoSjTHGmKAKSaIRkT4ikiMiX7rP8X7qVYvIRvex\n1Kt8pohsEJHNIvKciES45VeKyMfu4z0RGevruMYYY9pPqFo0i4DlqnoysNx978tRVR3nPuYAiEgY\n8BxwmaqOAvKBa9z6XwPTVXUM8HPgyWB+CWOMMU0LVaK5ECdZ4D5f1Ix9+wIVqvqF+z4HuBhAVd9T\n1QNu+VpgUBvEaowxphUCWmEzCPqp6i4AVd0lIkl+6sWISC5QBWSp6t+BfUCkiKSrai4wDxjsY98b\ngLeCELsxxnRKZZVl7C3b6zyOOs+n9T2NSQMmBfVzg5ZoRGQZ4GvKmnuacZgUVS0UkaHAChH5RFW3\nichlwEMiEg1k4yQi788+GyfRfKuR+G4EbgRISbFVqY0xnVdlTSXFR4vZU7aHorKi2mfvhFJUVkRJ\nZUmDfa877brOm2hUdba/bSKyR0QGuK2ZAcBeP8codJ+3i8gqYDywTVXfB6a6x8oETvE69hjgKeA8\nVS1uJL4nca/hpKen24qhxpgOR1U5WHGwrhVSL3F4yvaX70frrVMZIREkxiaSGJvISb1PYvKAySTF\nJtU+EmMT6Rfbj+6R3YP+PULVdbYU5wJ+lvv8j/oV3JFoZapaISIJwBRgsbstSVX3ui2ahcAv3fIU\n4K/A1V7XcIwxpsPxdGMVHa3XAnEfRUed95U1lQ327RPTx0kW3RIZ2XfkcQnEUx4fE0+YdIw7WEKV\naLKAV0TkBqAAuARARNKB76vqfGAE8ISI1OAMWshS1S3u/gtE5AK3/DFVXeGW/xRnsMDvRQSgSlXT\n2+tLGWOMpxvruFaIj4TiqxsrNiK2NlmMTxpf2+pI7JZ4XBKJDI8MwTdrOVG1XqP09HTNzc0NdRjG\nmA6sfjeWv5ZIU91Y9RNHe3djtSURWR/IH/OhatEYY0yHUVZZVttV5asl4nntrxvLkzg83Vj1E0pH\n6sYKBUs0xpguy7sbq3Y0Vr2EUlRWxJHKIw327RbRjX6x/UiKTWJc0jin9dHt+GshCd0SiAqPCsE3\n61ws0RhjOh1V5VDFIb+Jw1NefLTYZzdWQmwCSbFJDO01lDMHnFnXAolNrE0ocVFxIfp2XY8lGmNM\nh3K06qjfxOFddqzmWIN946Pja7uuRvYdeVzi8JT3ielzQndjhYIlGmNMu6iqqWLf0X0NbiT0Th57\ny/Y22o2VGJvodGN1O/4iemJsIondEq0bq4OyRGOMaZWmurE8CaXRbqxuSQzpNYQzBpxx/D0hbkLp\nHtkd95YF0wlZojHG+NXabixP19XwPsNr7wHxvhZi3VgnBks0xpyAWtuN5WlxjE0ce/xFdK+bCq0b\ny3hYojGmC/F0Y3nPh+VrgkVf3VjhEk5CtwT6xfYjrVfacd1Y3i2RuMg468YyzWKJxphO4mjV0UZn\n5vWU++rG6h3du/bi+fA+wxvcmZ4Um0R8dDzhYeEh+Gamq7NEY0wHtO/oPpbnL2f1ztXsPLKTvUf3\ncuSY/26sxG6JjE0c22BKk8RuzrQn0eHRIfgWxjgs0RjTQewt28uy/GVk52ezYc8GFCW1Zyon9T6J\n0/ufTr/uDefIsm4s0xlYojEmhHaX7iYnP4ec/Bw+2vsRACf1Pombx95MRmoGw3oPs0RiOj1LNMa0\nsx1HdrAsfxk5+Tl8vO9jAE6NP5Vbx9/K7NTZDO01NMQRGtO2LNEY0w4KDheQnZ9NTn4OW4qdZZVG\n9h3J7RNuJyM1g9SeqSGO0JjgsURjTJB8fehrcvJzyM7L5vMDnwMwJmEMP5r4I2anzmZQj0GtOn5l\nZSU7duygvLy8LcI1xq+YmBgGDRpEZGTLFlyzRGNMG1FVth3c5iSX/Gy+OvgVAOOTxvPj03/M7JTZ\nDIgb0Gaft2PHDnr06EFaWppdxzFBo6oUFxezY8cOhgwZ0qJjWKIxphVUlS8OfFHbLfb1oa8RhAn9\nJrDojEXMTplNv+79gvLZ5eXllmRM0IkIffv2paioqMXHsERjTDOpKlv2byEnzxktVnCkgDAJ4/R+\np3Pl8CuZlTqLhG4J7RKLJRnTHlp7nlmiMSYAqsrmfZtru8V2luwkXMKZNGAS1426jpkpM+kT0yfU\nYRrTIdm0qcb4UaM1bNy7kcUfLuac187hijev4IXPXmBIryH87KyfserSVTyR8QTzTpl3QiaZgwcP\n8vvf/75F+z788MOUlZW1Oobc3Fxuu+22Vh/H49prr+XVV19tUF5YWMi8efMAWLVqFRdccIHP/dPS\n0ti3b1+bxeMxf/58tmzZ0mgdf7Hn5eXxpz/9qdF9q6urGT9+vN/v1VrWojHGS3VNNR/t/Yic/ByW\n5S9j79G9RIZFMiV5Cj8c/0OmD5pOr+heoQ6zQ/AkmltuuaXZ+z788MNcddVVxMbGtiqG9PR00tPT\nW3WMQCQnJ/v8R7y9PPXUUy3e15NorrjiCr91HnnkEUaMGMHhw4db/DmNsRaNOeFV1VSxbtc6frH2\nF8z6yyyu+/d1vPbla4xOHE3W1CxWf3c1v531W+YMm2NJxsuiRYvYtm0b48aNY8GCBfz617/m9NNP\nZ8yYMdx7770AlJaWcv755zN27FhGjRrFyy+/zKOPPkphYSFnn302Z599tt/jx8XFsXDhQiZOnMjs\n2bNZt24dM2bMYOjQoSxduhQ4vnVx3333cf3119fWefTRRxuN//nnn2fMmDGMHTuWq6++urZ89erV\nnHXWWQwdOrQ2ueTl5TFq1KgGxyguLiYzM5Px48dz0003oaoN6ngsXry4NqY777yTmTNnArB8+XKu\nuuoqALKzs5k8eTITJkzgkksuoaSkBIAZM2aQm5sLwNNPP80pp5zCjBkz+M///E9++MMfNhr7okWL\neOeddxg3bhwPPfRQg7h27NjBG2+8wfz58xv9vVrDWjTmhFRZU8mHuz8kJz+HFQUr2F++n24R3Zg6\ncCoZaRlMGziN2MjW/bXdnu7/56dsKWzbv0ZHJvfk3u+c5nd7VlYWmzdvZuPGjWRnZ/Pqq6+ybt06\nVJU5c+awevVqioqKSE5O5o033gDg0KFD9OrViwcffJCVK1eSkOB/0ERpaSkzZszggQceYO7cufzk\nJz8hJyeHLVu2cM011zBnzpwG+2zdupWVK1dy5MgRTj31VG6++Waf9358+umn/PKXv2TNmjUkJCSw\nf//+2m27du3i3XffZevWrcyZM6e2y8yX+++/n29961v89Kc/5Y033uDJJ5/0W3fatGn85je/4bbb\nbiM3N5eKigoqKyt59913mTp1Kvv27eMXv/gFy5Yto3v37jzwwAM8+OCD/PSnP609RmFhIT//+c/Z\nsGEDPXr0YObMmYwdO7bR2LOysliyZAmvv/567THmz5/Pm2++CcAdd9zB4sWLOXKk4aStbcUSjTlh\nVFZXsnbXWie5fLOCQxWHiI2IZfqg6WSkZTAleUqnSi4dSXZ2NtnZ2YwfPx6AkpISvvzyS6ZOncpd\nd93FwoULueCCC5g6dWrAx4yKiuLcc88FYPTo0URHRxMZGcno0aPJy8vzuc/5559PdHQ00dHRJCUl\nsWfPHgYNanhj7IoVK5g3b15touvTp+4a20UXXURYWBgjR45kz549jca4evVq/vrXv9Z+dnx8vN+6\nEydOZP369Rw5coTo6GgmTJhAbm4u77zzDo8++ihr165ly5YtTJkyBYBjx44xefLk446xbt06pk+f\nXhvvJZdcwhdffNGs2JOTk2uTzOuvv05SUhITJ05k1apVjX7X1rBEY7q0iuoK3i98n5z8HFYWrORI\n5RHiIuOYMXgGGakZnJV8FjERMaEOs9Uaa3m0B1Xl7rvv5qabbmqwbf369bz55pvcfffdZGZmHvcX\nemMiIyNrh9WGhYURHR1d+7qqqsrnPp46AOHh4X7rqarfIbvex2isK8wj0KG/kZGRpKWl8cwzz3DW\nWWcxZswYVq5cybZt2xgxYgTNydecAAAYTElEQVTbtm0jIyODP//5z36P0VQ8zY19zZo1LF26lDff\nfJPy8nIOHz7MVVddxYsvvhjQdwqUXaMxXU55VTnLC5azcPVCpr88nVtX3MrKb1YyM2Umv5v1O97+\n7tv8auqvmJkys0skmVDp0aNHbXfLOeecwx/+8Ifaawo7d+5k7969FBYWEhsby1VXXcVdd93Fhg0b\nGuwbCrNmzeKVV16huLgY4Lius+aYNm0af/zjHwF46623OHDgQJP1lyxZwrRp05g6dSqPP/4448aN\nQ0Q488wzWbNmDV995cwoUVZWdlxrBeCMM87g7bff5sCBA1RVVfHaa681GWNjv/WvfvUrduzYQV5e\nHi+99BIzZ85s8yQD1qIxXURZZRnv7nyXnPwc3t7xNkerjtIruhfnpJ1DRmoGk/pPIjK8ZfM0Gd/6\n9u3LlClTGDVqFOeddx5XXHFFbVdPXFwcL774Il999RULFiwgLCyMyMhIHnvsMQBuvPFGzjvvPAYM\nGMDKlSvbPfbTTjuNe+65h+nTpxMeHs748eN59tlnm32ce++9l8svv5wJEyYwffp0UlJSGq0/depU\nfvnLXzJ58mS6d+9OTExMbXdiYmIizz77LJdffjkVFRUA/OIXv+CUU06p3X/gwIH813/9F5MmTSI5\nOZmRI0fSq1fjA1TGjBlDREQEY8eO5dprr+W73/3ucddo2oME0rxq8w8V6QO8DKQBecClqtrgTwER\nqQY+cd8WqOoct3wmsASIAtYDN6hqlYhcCPwcqAGqgDtU9d2m4klPT1fPiA7TeZRWlrJ6x2py8nN4\nZ8c7lFeX0yemD7NSZpGRmkF6/3Qiw7pucvnss88YMWJEqMMw7aykpIS4uDiqqqqYO3cu119/PXPn\nzg365/o630Rkvao2Ob48VC2aRcByVc0SkUXu+4U+6h1V1XHeBSISBjwHzFLVL0TkZ8A1wNPAcmCp\nqqqIjAFeAYYH84uY9nXk2BFWfbOKnPwc1uxcw7GaYyR0S+Ciky4iMy2TCUkTbN1706Xdd999LFu2\njPLycjIzM7noootCHVKTQpVoLgRmuK+fA1bhO9H40heoUFVP52UOcDfwtKqWeNXrDrR/c820uUMV\nh2qTy3uF71FZU0lSbBKXnnopGakZjE0ca8mlE5s0aVJtV5HHCy+8wOjRo1t97OLiYmbNmtWgfPny\n5fTt27fVxw/FZy5ZsqTVx2hvoUo0/VR1F4Cq7hKRJD/1YkQkF6cbLEtV/w7sAyJFJF1Vc4F5wGDP\nDiIyF/gVkAScH8wvYYLnQPkBVn6zkuz8bD4o/IAqrWJA9wFcPvxyMlIzGJM4hjCxsSxdwQcffBC0\nY/ft25eNGzcG7fgd5TM7uqAlGhFZBvT3semeZhwmRVULRWQosEJEPlHVbSJyGfCQiEQD2TiJCABV\n/RvwNxGZhnO9Zraf+G4EbgSavIBn2kfx0WKWFywnJz+HD3d/SLVWMyhuEFefdjWZqZmc1vc0m63Y\nmE4oaIlGVX3+Aw8gIntEZIDbmhkA7PVzjEL3ebuIrALGA9tU9X1gqnusTOAUH/uuFpFhIpKgqg1m\nuVPVJ4EnwRkM0OwvaNpEUVkRywuWk52fzfo966nRGlJ7pnL9qOvJSM1geJ/hllyM6eRC1XW2FOcC\nfpb7/I/6FUQkHihT1QoRSQCmAIvdbUmqutdt0SwEfumWn4STiFREJuCMSitujy9kAre7dLeTXPKy\n+WjvRyjKkF5D+M/R/0lGaganxJ9iycWYLiRUiSYLeEVEbgAKgEsARCQd+L6qzgdGAE+ISA3OjaVZ\nquqZJ3uBiFzglj+mqivc8ouB74lIJXAU+K6GYvy2aaCwpJCcfGehsE1FmwA4Of5kbh53M5mpmQzr\nPSzEERpjgkZVT/jHxIkT1bS9gsMF+vQnT+tl/7xMRz07Skc9O0ovWXqJPrnpSd1+cHuow+v0tmzZ\nEtLPP3DggP7ud79r0b4PPfSQlpaWtjqGDz/8UG+99dZWH8fjmmuu0b/85S8Nynfu3KkXX3yxqqqu\nXLlSzz//fJ/7p6amalFRUZvF43HDDTfop59+2mgdf7F//fXX+sc//tHvfqmpqTpq1CgdO3asNvZv\noa/zDcjVAP6NtZkBTJvKO5THsoJlZOdl89n+zwA4re9p3DHhDjJTMxncc3ATRzCdha1H036CvR5N\nUzNpt5aNDzWttv3gdh7f9DgXL72Y7/z9Ozyy4REiwyO5K/0u/nXxv3jpgpe4YfQNlmS6GFuPpmus\nR9MerEVjmk1V+fLgl841l7wcth3ahiCMTxrPwtMXMjt1Nv27+xrZboLmrUWw+5Om6zVH/9FwXpbf\nzbYeTddYj0ZEyMzMRES46aabuPHGG/1+h5ayRGMCoqp8fuBzsvOyycnPIe9wHmESxsR+E/nu8O8y\nK2UWSbH+7rs1XZ2tR9M516MBZ6mA5ORk9u7dS0ZGBsOHD2fatGmNfu/mskRj/FJVthRvITvfSS7f\nHPmGcAknvX86V4+8mpkpM0noFrx+XdMMjbQ82oPaejRN6ojr0YCTeACSkpKYO3cu69ata/NEY9do\nzHFqtIZNRZtY8uESzn3tXC574zKe//R5UnqkcN/k+1hx6QqeynyKS0+91JLMCc7Wo+n869GUlpbW\nbistLSU7O9vntajWshaNoUZr2Lh3Y+19LnvK9hARFsFZyWdx87ibOXvw2fSKbnzNC3PisfVoOv96\nNHv27KldYqCqqoorrriitruyLYVkPZqO5kRcj6a6ppoNezeQk5/DsvxlFB0tIiosiikDp5CRmsH0\nwdPpGdUz1GGaRth6NCcmW4/GdGhVNVXk7sklJy+HZQXL2F++n+jwaKYOnFqbXLpHdg91mMaYRth6\nNKbDqaypZN2udeTk57C8YDkHKw7SLaIb0wZNIyM1g6kDpxIb2bqb5oxpDVuPpnlsPRrTIRyrPsba\nXWvJzstm5TcrOXzsMN0juzN90HQyUzM5a+BZdIvoFuowjQFsPZoTgSWaLqKiuoL3dr5HTn4Oq75Z\nxZHKI/SI7MGMwTPITMtkcvJkosOjmz6QMca0MUs0ndjRqqOs2bmG7Pxs3v7mbcqqyugZ1ZNZqbPI\nSM3gzAFnEhUeFeowjTEnOEs0nUxZZRmrd64mJy+Hd3a+w9Gqo8RHx3PekPPITM3k9AGnExnWcMoN\nY4wJFUs0nUDJsRJW71hNTn4O7+58l/LqcvrG9GXOsDlkpGYwsd9EIsLsP6UxpmOymQE6qMPHDvPP\nbf/k1hW3Mv3l6Sx8ZyGbijYx9+S5/OGcP7D8kuX85MyfMGnAJEsyJiQ8ywS0xMMPP0xZWVmrY8jN\nzeW2225r9XE8rr32Wp/LARQWFtZOruk9Y3R9aWlp7NvXYOX4Vps/fz5btmxptI6/2D3LBPhz8OBB\n5s2bx/DhwxkxYgTvv/9+q+Otz/6F6kAOVRxiRcEKcvJzeH/X+1TVVNEvth+XnnopmWmZjE0cS5jY\n3wamY7D1aNpPMNejuf322zn33HN59dVXOXbsWJv8AVCfJZoQ21++vza5rNu1jiqtYmDcQK4acRUZ\nqRmMShhlycU06YF1D7B1/9Y2PebwPsNZeMZCv9u916PJyMggKSmJV155hYqKCubOncv9999PaWkp\nl156KTt27KC6upr//u//Zs+ePbXr0SQkJPidgiYuLo4f/OAHLFu2jPj4eP7nf/6HH//4xxQUFPDw\nww8zZ84cVq1aVTsF/n333UdBQQHbt2+noKCAO+64o9HWzvPPP8+SJUsQEcaMGcMLL7wAODMyP/jg\ng+zevZvFixczb9488vLyuOCCC9i8efNxxyguLubyyy+nqKiIM844o8n1aGJiYrjtttu488472bRp\nEytWrGD58uU888wzvPjii2RnZ3PvvfdSUVHBsGHDeOaZZ4iLi2PGjBksWbKE9PR0nn76aR544AGS\nk5M5+eSTiY6O5n//93/9xr5o0SI+++wzxo0bxzXXXMOdd95ZG9Phw4dZvXp17fQ7UVFRREW1/QAi\n+xcsBPYd3ccrn7/C/H/P5+xXzub+9+/nmyPf8L3TvsdLF7zEW//xFj9K/xFjEsdYkjEdVlZWFsOG\nDWPjxo1kZGTw5Zdfsm7dOjZu3Mj69etZvXo1//rXv0hOTmbTpk1s3ryZc889l9tuu43k5GRWrlzZ\n6DxnnvVo1q9fT48ePWrXo/nb3/7mdwborVu38u9//5t169Zx//33U1lZ6bOeZz2aFStWsGnTJh55\n5JHabZ41XV5//XUWLVrU6G/gWY/mo48+Ys6cORQUFPitO23aNN555x3A6fIrKSnxux7Nhg0bSE9P\n58EHHzzuGJ71aNauXUtOTg5btx7/x4Wv2LOyspg6dSobN27kzjvvpLCwkG9/+9sAbN++ncTERK67\n7jrGjx/P/PnzKS0tbfQ7t4S1aNrJ3rK9LMtfRk5+Duv3rEdR0nqmccOoG8hMy+TU+FMDnm7cmPoa\na3m0B1uPpnOuR1NVVcWGDRv47W9/y6RJk7j99tvJysri5z//eaPfu7ks0QTR7tLdtTMib9y7EUU5\nqfdJfH/s98lIzeCk3idZcjFdgq1H07SOuB7NoEGDGDRoEJMmTQKoXZGzrVm/TBvbWbKTZzc/y5Vv\nXEnGqxks/nAxpZWl/GDcD/jHhf/gbxf+jVvG3cLJ8SdbkjGdmq1H0/nXo+nfvz+DBw/m888/B5z5\n2EaOHNnkMZvLWjRtoOBwQW3L5dPiTwEY0WcEt0+4ndkps0nrlRbaAI0JAluPpvOvRwPw29/+liuv\nvJJjx44xdOhQnnnmmWb/Dk2x9Who+Xo0a3et5Te5v6kd7TM6YTQZqRnMTp3N4B6D2zpMY45j69Gc\nmGw9mhNMXGQc0eHRLEhfwOzU2STHJYc6JGNMF2fr0ZxgRiWM4sVvvxjqMIzp1Gw9muax9WiMMaaZ\nbD2ars9GnRnTidk1VtMeWnuehSTRiEgfEckRkS/dZ593OYlItYhsdB9LvcpnisgGEdksIs+JSES9\n/U53950X7O9iTKjExMRQXFxsycYElapSXFxMTExMi48Rqq6zRcByVc0SkUXue1+3Nh9V1XHeBSIS\nBjwHzFLVL0TkZ8A1wNPu9nDgAeDfwfwCxoTaoEGD2LFjB0VFRaEOxXRxMTExPmdYCFSoEs2FwAz3\n9XPAKnwnGl/6AhWq6rmTKQe4GzfRALcCrwGnt0WgxnRUkZGRDBkyJNRhGNOkUF2j6aequwDc5yQ/\n9WJEJFdE1oqIZwzfPiBSRDxjt+cBgwFEZCAwF3i8qQBE5Eb32Ln2F6ExxgRP0Fo0IrIM6O9j0z3N\nOEyKqhaKyFBghYh8oqrbROQy4CERiQayAc+ERg8DC1W1uqnpXVT1SeBJcG7YbEZMxhhjmiFoiUZV\nZ/vbJiJ7RGSAqu4SkQHAXj/HKHSft4vIKmA8sE1V3wemusfKBDxzNKQDL7lJJgH4tohUqerf2+hr\nGWOMaaaQTEEjIr8Gir0GA/RR1R/XqxMPlKlqhYgkAO8DF6rqFhFJUtW9bovmTeCXqrqi3v7PAq+r\napPL4olIEZDfwq+TgNOd19F01Lig48ZmcTWPxdU8XTGuVFVNbKpSqAYDZAGviMgNQAFwCYB73eX7\nqjofGAE8ISI1ONeSslTVs2j2AhG5wC1/rH6Saa5Afih/RCQ3kLl+2ltHjQs6bmwWV/NYXM1zIscV\nkkSjqsVAgzkaVDUXmO++fg/wOQeFqi4AFjTxGde2OlBjjDGtZjMDGGOMCSpLNK33ZKgD8KOjxgUd\nNzaLq3ksruY5YeOy9WiMMcYElbVojDHGBJUlmkaISG8ReVVEtorIZyIyud52EZFHReQrEflYRCZ4\nbbvGnTT0SxG5pp3jutKN52MReU9ExnptyxORT9yJSpu/rGjr4pohIoe8Jkr9qde2c0Xkc/e3XNTO\ncS3wimmzOyFrH3dbUH4vETnV6zM3ishhEbmjXp12P78CjKvdz68A42r38yvAuNr9/HKPfaeIfOp+\n5p9FJKbe9mgRedn9TT4QkTSvbXe75Z+LyDmtDkZV7eHngTMP23z3dRTQu972bwNvAQKcCXzglvcB\ntrvP8e7r+HaM6yzP5wHneeJy3+cBCSH6vWbg3NtUf79wYBsw1N1vEzCyveKqV/c7wIr2+L3qff/d\nOPckhPz8CiCukJxfAcQVkvOrqbhCcX4BA4GvgW7u+1eAa+vVuQV43H19GfCy+3qk+xtFA0Pc3y68\nNfFYi8YPEekJTMOdrFNVj6nqwXrVLgSeV8daoLc4Mx2cA+So6n5VPYAz8ee57RWXqr7nfi7AWqDl\n0662YVyNOAP4SlW3q+ox4CWc3zYUcV0O/LktPrsZZuHMeFH/puF2P78CiSsU51cgcTUiaOdXC+Jq\nz/MrAugmzjIqsUBhve0X4vwRBvAqMEtExC1/SVUrVPVr4Cuc37DFLNH4NxQoAp4RkY9E5CkR6V6v\nzkDgG6/3O9wyf+XtFZe3G3D+KvZQIFtE1ovIjW0UU3Pimiwim0TkLRE5zS3rEL+XiMTi/IP9mldx\nsH4vb5fh+x+fUJxfgcTlrb3Or0Djau/zK9C42vX8UtWdwBKcG+J3AYdUNbtetdrfRVWrgEM4s+O3\n+e9lica/CGACzswD44FSnHVzvPmauVMbKW+vuJzgRM7G+YfAewmGKao6AafL4wciMq0d49qA060w\nFvgt4JmDrkP8XjjdGmtUdb9XWbB+LwBEJAqYA/zF12YfZcE+vwKJy1OnPc+vQOIKxfkVSFwe7XZ+\niTOF14U4XV/JQHcRuap+NR+7BuX8skTj3w5gh6p6FjR/FecfrPp1Bnu9H4TTPPVX3l5xISJjgKdw\n5ocr9pRr3USle4G/0comcXPiUtXDqlrivn4TZ7mHBDrA7+Vq8BdpEH8vj/OADaq6x8e2UJxfgcQV\nivOrybhCdH41GZeX9jy/ZgNfq2qRqlYCf8W5tuat9ndxu9d6AfsJwu9licYPVd0NfCMip7pFs4At\n9aotBb7njg46E6d5ugtndc9MEYl3/7LIpI1W/AwkLhFJwTmxrta6BeIQke4i0sPz2o1rczvG1d/t\nA0ZEzsA5/4qBD4GTRWSI+5fhZTi/bbvE5cbTC5gO/MOrLGi/l5fG+uzb/fwKJK5QnF8BxtXu51cg\ncbnxtPf5VQCcKSKx7m8yC/isXp2lOKsTg7Ou1wp1RgMsBS5zR6UNAU4G1rUqmrYY4dBVH8A4IBf4\nGKcZHg98H2fiT3CamL/DGZXxCZDute/1OBfRvgKua+e4ngIOABvdR65bPhRnNMkm4FPgnnaO64fu\n527CuYh8lte+3wa+cH/Ldo3LrXMtzgVQ7/2C/XvF4vxD2MurrCOcX03FFarzq6m4QnV+NRpXCM+v\n+4GtOMnrBZxRZD8D5rjbY3C6+r7CSSRDvfa9x/2tPgfOa20sNjOAMcaYoLKuM2OMMUFlicYYY0xQ\nWaIxxhgTVJZojDHGBJUlGmOMMUFlicYYY0xQWaIxphNxp5VPaOG+14pIclscy5jmsERjzInjWpx5\nr4xpV5ZojGkBEUkTZyG1p9yFpf4oIrNFZI04i5Gd4T7ec2eNfs8zDY6I/D8R+YP7erS7f6yfz+kr\nItnuMZ7Aa8JDEblKRNaJs2jWEyIS7paXiMhvRGSDiCwXkUQRmQekA39063dzD3OrW+8TERkezN/M\nnLgs0RjTcicBjwBjgOHAFcC3gLuA/8KZ/mOaOrNG/xT4H3e/h4GTRGQu8Axwk6qW+fmMe4F33WMs\nBVIARGQE8F2c2X/HAdXAle4+3XEmeJwAvA3cq6qv4kzDc6WqjlPVo27dfW69x9y4jWlzEaEOwJhO\n7GtV/QRARD4FlquqisgnQBrObLjPicjJONOsRwKoao2IXIsz99oTqrqmkc+YBvyHu98bIuJZcGwW\nMBH40J1Hshuw191WA7zsvn4RZwJMfzzb1ns+x5i2ZonGmJar8Hpd4/W+Buf/rZ8DK1V1rjjrsa/y\nqn8yUEJg10x8TUgowHOqencL9/fwxFyN/XtggsS6zowJnl7ATvf1tZ5Cd8r4R3BaK33d6yf+rMbt\nEhOR83BmngZYDswTkSR3Wx8RSXW3heFM+w5Od9677usjQI9WfB9jWsQSjTHBsxj4lYisAcK9yh8C\nfq/OWi43AFmehOHD/cA0EdmAs15JAYCqbgF+grMM8MdADjDA3acUOE1E1gMzcaaGB3gWeLzeYABj\ngs6WCTCmixGRElWNC3UcxnhYi8YYY0xQWYvGmA5ARK4Dbq9XvEZVfxCKeIxpS5ZojDHGBJV1nRlj\njAkqSzTGGGOCyhKNMcaYoLJEY4wxJqgs0RhjjAmq/w+M5LppMy3JMwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14d43a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "    pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( '- Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
